package com.zte.cl.parser;

import java.util.ArrayList;
import java.util.List;

import org.apache.log4j.Logger;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Selectable;

import com.zte.cl.model.News;
import com.zte.cl.model.SeedPage;
import com.zte.cl.entity.Field;
import com.zte.cl.entity.PageTemplate;
import com.zte.cl.parser.enums.PageType;
import com.zte.cl.parser.utils.SelectableUtil;
import com.zte.cl.utils.ConstantUtil;
import com.zte.cl.utils.DateUtil;
import com.zte.cl.utils.ExtractUtil;
import com.zte.cl.utils.StringUtil;

/**
 * 种子页解析器 PageProcessor负责解析页面，抽取有用信息，以及发现新的链接
 * 
 * @author Administrator
 * 
 */
public class SeedPageProcessor implements PageProcessor {

	private final static Logger log = Logger.getLogger(SeedPageProcessor.class);

	private Site site;
	private SeedPage seedPage;

	@Override
	public void process(Page page) {
		String url = page.getRequest().getUrl();
		PageTemplate wp = PageTemplateUtil.findPageTemplate(url, seedPage.getPageType()); // 列表页面
		if(wp==null){
			 wp = PageTemplateUtil.findPageTemplateByUrlPrefix(url, seedPage.getPageType()); //url前缀匹配
		}
		if (wp != null) {
			Selectable listSt = null;
			if (wp.getType() == PageType.rssPage) {
				listSt = SelectableUtil.getSelectable(page.getRawText(), wp.getExprs());
			} else {
				listSt = SelectableUtil.getSelectable(page.getHtml(), wp.getExprs());
			}
			List<String> linkList = listSt.all(); // 单个文章html片段
			List<Field> fields = wp.getFields();
			List<News> newsList = new ArrayList<News>();
			if (linkList != null && fields != null) {
				for (String str : linkList) {
					News news = new News();
					boolean check = ExtractUtil.extractToNews(str, fields, news);
					if ((!StringUtil.isEmpty(news.getContent()) || !StringUtil.isEmpty(news.getPageUrl())) && check) {
						news.setSource(seedPage.getName());
						news.setInsertTime(DateUtil.getCurrentDate());
						news.setChannelId(seedPage.getChannelId());
						news.setDeptId(seedPage.getDeptId());
						news.setFlag(seedPage.getFlag());
						newsList.add(news);
					} else {
						log.warn("此条news无效, content:" + news.getContent() + ", pageUrl:" + news.getPageUrl() + " url:" + url + ", check:" + check);
					}
					
				}
			}
			page.putField("seedPageType", seedPage.getPageType());
			page.putField("newsList", newsList);
			log.info(String.format("name=%s, url=%s,channel_id=%s, 发现 %s 条新闻", seedPage.getName(), seedPage.getUrl(),
					seedPage.getChannelId(), newsList.size()));

		} else {
			log.warn(String.format("seed没有找到对应的template信息，name=%s,url=%s,page_type=%s", seedPage.getName(),
					seedPage.getUrl(), seedPage.getPageType()));
		}
	}

	public SeedPageProcessor(SeedPage seedPage) {
		this.seedPage = seedPage;
	}

	@Override
	public Site getSite() {
		if (this.site == null) {
			this.site = Site.me().setSleepTime(ConstantUtil.SITE_SLEEP_TIME).setUserAgent(ConstantUtil.SITE_USER_AGENT);
		}
		return this.site;
	}

}
